#coding:utf-8
import urllib, urllib2, cookielib
import re
import os
import Queue
import threading
import time
import socket
from multiprocessing.dummy import Pool as ThreadPool
import sys
from PyQt4 import QtGui, QtCore

socket.setdefaulttimeout(20)
queue = Queue.Queue()
name_dic = {}
email_dic = {}

class UserForm(QtGui.QWidget):
    
    def __init__(self):
        super(UserForm, self).__init__()
        self.lbl1 = QtGui.QLabel("username")
        self.qle1 = QtGui.QLineEdit(self)
        self.lbl2 = QtGui.QLabel("password")
        self.qle2 = QtGui.QLineEdit(self)
        self.nameButton= QtGui.QPushButton("ok")  
        self.connect(self.nameButton, QtCore.SIGNAL("clicked()"), self.msgbox)  
        layout=QtGui.QGridLayout()  
        
        layout.addWidget( self.lbl1,1,0)  
        layout.addWidget( self.qle1,1,1)  
        layout.addWidget( self.lbl2,3,0)  
        layout.addWidget( self.qle2,3,1) 
        layout.addWidget( self.nameButton,4,2)  
        self.setLayout(layout)  
        self.show()
        
    def msgbox(self):
        username = self.qle1.text()
        password = self.qle2.text()
        self.hide()
        main(username, password)

class TigerMail(threading.Thread):

    def __init__(self, queue, username, password):
        threading.Thread.__init__(self)
        self.queue = queue
        self.mainlist = []
        self.name = username
        self.passwd = password
        self.diry = os.getcwd() + '\\' + self.name
        if not os.path.exists(self.diry):
            os.makedirs(self.diry)
        self.login()
        
    def run(self):
        while not exit_flag.is_set():
            url = self.queue.get()
            self.GetCont(url)
            print '**********Finished: %s **********' % url
            time.sleep(1)
            self.queue.task_done()

    def login(self):
        try:
            self.cj = cookielib.LWPCookieJar()
            login_first = 'https://www.introducer.westpac.net.au/inn/secure/Logout.do'
            login_url = 'https://www.introducer.westpac.net.au/inn/secure/Login.do'
            self.opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self.cj))
            urllib2.install_opener(self.opener)

            headers = {'Accept-Language': 'zh-CN',
                       'User-Agent': 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)',
                       'Content-Type': 'application/x-www-form-urlencoded',
                       'Host': 'www.introducer.westpac.net.au',
                       'Connection': 'Keep-Alive',
                        }
            req = urllib2.Request(login_first, headers=headers,)
            content = self.opener.open(req).read()

            PostData = {'userID': self.name, 'password': self.passwd, 'command': 'login'}
            login_data = urllib.urlencode(PostData)
            headers = {'Referer': 'https://www.introducer.westpac.net.au/inn/secure/Logout.do',
                       'Accept-Language': 'zh-CN',
                       'User-Agent': 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)',
                       'Content-Type': 'application/x-www-form-urlencoded',
                       'Host': 'www.introducer.westpac.net.au',
                       'Connection': 'Keep-Alive',
                       'Cache-Control': 'no-cache'
                        }
            req = urllib2.Request(login_url, login_data, headers=headers,)
            conn = self.opener.open(req)
            html = conn.read()
            if 'Application progress' in html:
                print 'Login OK'
            # for index, cookie in enumerate(self.cj):
                # print '[', index, ']', cookie
        except Exception as e:
            print e, 'Login() error'
            time.sleep(15)
            self.login()
        finally:
            try:
                conn.close()
            except NameError:
                pass
            
    def GetList(self):
        try:
            ListUrl = 'https://www.introducer.westpac.net.au/inn/secure/StartSearch.do'
            headers = {'Referer': 'https://www.introducer.westpac.net.au/inn/secure/Login.do',
                       'Accept-Language': 'zh-CN',
                       'User-Agent': 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)',
                       'Content-Type': 'application/x-www-form-urlencoded',
                       'Host': 'www.introducer.westpac.net.au',
                       'Connection': 'Keep-Alive',
                        }
            req = urllib2.Request(ListUrl, headers=headers,)
            conn = self.opener.open(req)
            html = conn.read()
            url_list_pattern = re.compile(r'<a href="(RetrieveApplicationDetail.do\?appNo=\d+)">\s+(.*?)\s+</a>', re.S)
            url_list = url_list_pattern.findall(html)
            self.mainlist.extend(url_list)
            if 'name="command" value="Next"' in html:
                print 'Goto Next Page>>>>>>>>>>>>>>>>>>>>>>>'
                self.NextUrl()
        except Exception as e:
            print e, 'Getlist() error'
            time.sleep(15)
            print 'sleep 5s'
            self.GetList()
        finally:
            try:
                conn.close()
            except NameError:
                pass

    def NextUrl(self):
        try:
            next_url = 'https://www.introducer.westpac.net.au/inn/secure/ContinueSearch.do'
            PostData = {'searchType': '0', 'sortField': '0', 'command': 'Next'}
            login_data = urllib.urlencode(PostData)
            headers = {'Referer': 'https://www.introducer.westpac.net.au/inn/secure/StartSearch.do',
                       'Accept-Language': 'zh-CN',
                       'User-Agent': 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)',
                       'Content-Type': 'application/x-www-form-urlencoded',
                       'Host': 'www.introducer.westpac.net.au',
                       'Connection': 'Keep-Alive',
                       'Cache-Control': 'no-cache'
                        }
            req = urllib2.Request(next_url, login_data, headers=headers,)
            conn = self.opener.open(req)
            html = conn.read()
            url_list_pattern = re.compile(r'<a href="(RetrieveApplicationDetail.do\?appNo=\d+)">\s+(.*?)\s+</a>', re.S)
            url_list = url_list_pattern.findall(html)
            self.mainlist.extend(url_list)
            if 'name="command" value="Next"' in html:
                print 'Goto Next Page>>>>>>>>>>>>>>>>>>>>>>>'
                self.NextUrl()
            else:
                print 'All list urls geted'
        except Exception as e:
            print e, 'NextUrl() error'
            time.sleep(10)
            self.NextUrl()
        finally:
            try:
                conn.close()
            except NameError:
                pass

    def GetCont(self, url):
        try:
            headers = {'Referer': 'https://www.introducer.westpac.net.au/inn/secure/StartSearch.do',
                       'Accept-Language': 'zh-CN',
                       'User-Agent': 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)',
                       'Content-Type': 'application/x-www-form-urlencoded',
                       'Host': 'www.introducer.westpac.net.au',
                       'Connection': 'Keep-Alive',
                       'Cache-Control': 'no-cache'
                        }
            req = urllib2.Request(url, headers=headers,)
            conn = self.opener.open(req)
            html = conn.read()
            pattern = re.compile(r'''onClick="document.location.href='(.*?)'"''')
            comment_url = pattern.search(html)
            if comment_url:
                com_url = comment_url.group(1)
                self.GetComm(url, com_url)
            else:
                with open('err.html', 'w') as f:
                    f.write(html)
                print 'GetCont no group, Loged out, Wait for 15s .'
                print 'Put in queue: %s' % url
                self.queue.put(url)
                time.sleep(15)
                self.login()
        except Exception as e:
            print e, url, 'Getcont() error'
            print 'Put in queue: %s' % url
            self.queue.put(url)
        finally:
            try:
                conn.close()
            except NameError:
                pass


    def GetComm(self, Con_url, com_url):
        try:
            headers = {'Referer': Con_url,
                       'Accept-Language': 'zh-CN',
                       'User-Agent': 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)',
                       'Content-Type': 'application/x-www-form-urlencoded',
                       'Host': 'www.introducer.westpac.net.au',
                       'Connection': 'Keep-Alive',
                       'Cache-Control': 'no-cache'
                        }
            req = urllib2.Request(com_url, headers=headers,)
            conn = self.opener.open(req)
            html = conn.read()
            pa_name = re.compile(r'\d+-\d+-\d+')
            if pa_name:
                name = pa_name.search(html).group(0)
                name = self.diry + '\\' + name + '.html'
                with open(name, 'w') as f:
                    f.write(html)
                if 'value="Next"' in html:
                    # print 'Goto Next Page Comment>>>>>>>>>>>>>>>'
                    self.PostComm(Con_url, com_url, name)
                else:
                    print 'ALL Page Comment Geted'
            else:
                with open('err2.html', 'w') as f:
                    f.write(html)
                print 'Loged out , getcomm wait 15s.'
                print 'Put in queue: %s' % Con_url
                self.queue.put(Con_url)
                time.sleep(15)
                self.login()
            
        except Exception as e:
            print e, 'GetComm error', com_url
            print 'Put in queue: %s' % Con_url
            self.queue.put(Con_url)
        finally:
            try:
                conn.close()
            except NameError:
                pass


    def PostComm(self, Con_url, com_url, name):
        try:
            post_url = 'https://www.introducer.westpac.net.au/inn/secure/CommentsPaging.do'
            post_data = {'command': 'Next'}
            PostData = urllib.urlencode(post_data)
            headers = {'Referer': com_url,
                       'Accept-Language': 'zh-CN',
                       'User-Agent': 'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0)',
                       'Content-Type': 'application/x-www-form-urlencoded',
                       'Host': 'www.introducer.westpac.net.au',
                       'Connection': 'Keep-Alive',
                       'Cache-Control': 'no-cache'
                        }
            req = urllib2.Request(post_url, PostData, headers=headers)
            conn = self.opener.open(req)
            html = conn.read()
            with open(name, 'a') as f:
                f.write(html)
            if 'value="Next"' in html:
                print 'Goto Next Page Comment>>>>>>>>>>>>>>>'
                self.PostComm(Con_url, post_url, name)
            # else:
                # print 'ALL Page Comment Geted'
        except Exception as e:
            print e, 'PostComm error', Con_url
            print 'Put in queue: %s' % Con_url
            self.queue.put(Con_url)
        finally:
            try:
                conn.close()
            except NameError:
                pass
                
def form():
    app = QtGui.QApplication(sys.argv)
    ex = UserForm()
    sys.exit(app.exec_()) 
    
def main(username, password):
    username = str(username)
    password = str(password)
    global exit_flag
    exit_flag = threading.Event()
    exit_flag.clear()
    
    Tiger1 = TigerMail(queue, username, password)
    Tiger1.GetList()
    Tiger1.mainlist = [('https://www.introducer.westpac.net.au/inn/secure/' + x, y) for x, y in Tiger1.mainlist]
    print len(Tiger1.mainlist), 'files'
    
    for i in range(10):
        t = TigerMail(queue, username, password)
        t.start()
    
    for x, y in Tiger1.mainlist:
        queue.put(x)
    
    queue.join()
    exit_flag.set()
    print 'Download done !'
    os._exit(0)
    
if __name__ == '__main__':
    form()
    
    
    